package com.bestv.search.engine.accesslog;

import com.bestv.search.common.dataSource.DataSourceKeyUtil;
import com.bestv.search.common.model.AccessLog;
import com.bestv.search.common.model.TopKeyword;
import com.bestv.search.common.service.TopKeywordManager;
import com.bestv.search.common.util.StringConst;
import org.apache.log4j.Logger;
import org.dom4j.Document;
import org.dom4j.io.SAXReader;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;

import java.io.*;
import java.net.URLDecoder;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @author Justin Luo
 */
public class AccessLogAnalyzer {

    // Sets buffer size to 0.5M
    private static final int BUFFER_SIZE = 512000;
    private static final int MINUTE_INTERVAL = 15;
    private static final int MINUTE_SPLIT_TOTAL_COUNT = 1440 / MINUTE_INTERVAL;
    private static ApplicationContext context = new ClassPathXmlApplicationContext(new String[]{
            "applicationContext-compass.xml", "applicationContext-resources.xml", "applicationContext-service.xml"});
    private static int maxAnalyzeDays;
    private Logger logger = Logger.getLogger(AccessLogAnalyzer.class);
    private Hashtable<Date, List<AccessLog[]>> result;
    private String logsFolder;
    private Date beginDate;
    private String siteCode;
    private int serverNo;
    private TopKeywordManager topKeywordManager;
    private Map<String, Integer> allTopKeywordSet = new HashMap<String, Integer>();

    public AccessLogAnalyzer() {
        topKeywordManager = (TopKeywordManager) context.getBean("topKeywordManager");
    }

    private static String getDefaultLogsFolder() {
        // If don't config logs folder, use default folder that is
        // tomcat/logs
        String classesPath = AccessLogAnalyzer.class.getClassLoader().getResource(File.separator).getPath();
        return classesPath + "../../../../logs";
    }

    public Hashtable<Date, List<AccessLog[]>> getAccessLog(String siteCode, int serverNo, Date lastAnalyzeDate,
                                                           int maxAnalyzeDays, String logsFolder) {

        this.beginDate = lastAnalyzeDate;
        this.siteCode = siteCode;
        this.serverNo = serverNo;
        this.logsFolder = logsFolder == null || logsFolder.equals(StringConst.EMPTY) ? getDefaultLogsFolder()
                : logsFolder;
        AccessLogAnalyzer.maxAnalyzeDays = maxAnalyzeDays;

        this.result = new Hashtable<Date, List<AccessLog[]>>();

        // Gets topKeywords set

        List<TopKeyword> allTopKeywords = topKeywordManager.getAllTopKeywords(DataSourceKeyUtil.convert(this.siteCode));
        convertToMap(allTopKeywords);

        // Gets all access log files
        String[] files = getAllAccessLogFiles();
        if (files == null || files.length == 0) {
            logger.error("No access log file found: " + new Date());
            return result;
        }

        int threadCount = files.length;
        ThreadPoolExecutor threadPool = new ThreadPoolExecutor(threadCount, threadCount, threadCount, TimeUnit.SECONDS,
                new ArrayBlockingQueue<Runnable>(threadCount), new ThreadPoolExecutor.DiscardOldestPolicy());
        CountDownLatch latch = new CountDownLatch(threadCount);

        // Handle file one by one
        for (String fileName : files) {
            threadPool.execute(new AnalyzerTask(fileName, latch));
        }

        // Wait for all files were analyzed
        try {
            latch.await();
        } catch (InterruptedException e) {
            logger.error(e);
            e.printStackTrace();
        }

        threadPool.shutdown();
        logger.info("Getting access logs : " + result.size());

        return result;
    }

    // Gets an AccessLog by log text
    private AccessLog getAccessLog(String logText, Date date) {
        AccessLog accessLog = new AccessLog();
        String logFormat = ".*\\d{4}:(\\d{2}:\\d{2}:\\d{2}).*/engine/(.*)\\?q=(.*)";

        Pattern pattern = Pattern.compile(logFormat, Pattern.CASE_INSENSITIVE);
        Matcher matcher = pattern.matcher(logText);
        // Just collect record that matches with logFormat.
        if (matcher.matches()) {
            // Gets access type
            String action = matcher.group(2);
            if ("dotopn".equalsIgnoreCase(action) || "dosuggest".equalsIgnoreCase(action)
                    || "dosearch".equalsIgnoreCase(action) || "dohit".equalsIgnoreCase(action)) {
                // Gets access parameters
                String paras = matcher.group(3);
                int blankIndex = paras.indexOf(" ");
                paras = paras.substring(0, blankIndex);

                int type = getType(action, paras);
                if (type == -1) {
                    logger.error("invalid XML character at line " + paras + ", ignore it");
                    return null;
                }
                accessLog.setType(type);

                String time = matcher.group(1);

                SimpleDateFormat date_format = new SimpleDateFormat(StringConst.DATEFORMAT);
                SimpleDateFormat date_time_format = new SimpleDateFormat(StringConst.DATEFTIMEFORMAT);
                try {
                    String dateTime = date_format.format(date) + StringConst.SPACE + time;
                    Date accessTime = date_time_format.parse(dateTime);
                    accessLog.setAccessLowerTime(accessTime);
                } catch (Exception e) {
                    logger.error("invalid time format" + paras + ", ignore it");
                    e.printStackTrace();
                    return null;
                }
                accessLog.setDescription(getDescriptionByType(type));
                accessLog.setSiteCode(this.siteCode);
                accessLog.setServerNo(this.serverNo);
                accessLog.setLastUpdateTime(new Date());
            } else {
                accessLog = null;
            }
        } else {
            accessLog = null;
        }

        return accessLog;
    }

    /**
     * Gets all access log files that will be analyzed
     * <p/>
     * If beginDate isn't null, collect log files that "log files' date">=beginDate and "log files' date" < today.
     * <p/>
     * If beginDate is null, collect log files that "log files' date" < today and { today - "log files' date" <=
     * MaxAyalyzeDays}.
     *
     * @return File list.
     */
    private String[] getAllAccessLogFiles() {
        File dir = new File(logsFolder);

        FilenameFilter filter = new FilenameFilter() {
            public boolean accept(File dir, String name) {
                Calendar c = getCalendarFromFileName(name);
                // Just get files that match with pattern
                // "localhost_access_log\\.(\\d{4})-(\\d{2})-(\\d{2})\\.txt"
                if (c != null) {
                    Calendar today = Calendar.getInstance();
                    SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");

                    // Doesn't collect log file that "log files' date" >= today
                    if (c.getTime().after(today.getTime())
                            || dateFormat.format(c.getTime()).equals(dateFormat.format(today.getTime()))) {
                        return false;
                    }

                    // Doesn't collect log file that { today - "log files' date"
                    // > MaxAyalyzeDays}
                    today.add(Calendar.DAY_OF_YEAR, -AccessLogAnalyzer.maxAnalyzeDays);
                    if (today.getTime().after(c.getTime()))
                        return false;

                    // If beginDate is null, return true.
                    if (beginDate == null) {
                        return true;
                    } else {
                        // Return "log files' date">=beginDate
                        return beginDate.before(c.getTime())
                                || dateFormat.format(c.getTime()).equals(dateFormat.format(beginDate));
                    }
                } else {
                    return false;
                }
            }
        };

        return dir.list(filter);
    }

    /**
     * Gets type of access log.
     *
     * @param action - Possible values are dotopn|dosuggest|dosearch|dohit
     * @param paras  - Search parameters of requesting.
     * @return type - Access type to search engine.
     */
    private int getType(String action, String paras) {
        if (action.equals("dotopn")) {
            return 10;
        } else if (action.equals("dosuggest")) {
            return 20;
        } else if (action.equals("dohit")) {
            return 30;
        } else {
            // gets invalid xml character at sometime, set type to -1
            if (paras == null) {
                return -1;
            }

            // Gets search type, possibe values are ( 0为检索， 1 为首字母搜索，2为提示框搜索)
            String keyword = null;
            String paraFormat = ".*<keywords>(.*)</keywords>.*";
            try {
                paras = URLDecoder.decode(paras, "UTF-8");
            } catch (UnsupportedEncodingException e) {
                logger.error("Encoding para to UTF-8 failed " + paras);
                e.printStackTrace();
                return -1;
            }
            Pattern paraPattern = Pattern.compile(paraFormat, Pattern.CASE_INSENSITIVE);
            Matcher paraMatcher = paraPattern.matcher(paras);
            if (paraMatcher.matches()) {
                keyword = paraMatcher.group(1);
            }

            if (keyword == null || keyword.equals("")) {
                // Doesn't exist search key word, the search type should be "检索"
                return 0;
            } else {
                Integer count = allTopKeywordSet.get(keyword);
                // if keyword doesn't exist in Topkeyword, search type should be
                // "首字母搜索" or "检索"
                if (count == null) {
                    // if keyword match with pattern
                    // \\+genre:\\(.*\\)|\\+\\(.*\\), the search type is "检索"
                    String pattern = "\\+genre:\\(.*\\)|\\+\\(.*\\)";
                    if (isMatchPattern(keyword, pattern)) {
                        return 0;
                    } else {
                        return 1;
                    }
                } else {
                    // if the keyword exist in Topkeyword and its count equals
                    // to 1, the search type should be "首字母搜索"
                    if (count == 1)
                        return 1;
                        // searches type == "提示框搜索".
                    else
                        return 2;
                }
            }
        }
    }

    private String getDescriptionByType(int type) {
        switch (type) {
            case 0:
                return "检索搜索请求";
            case 1:
                return "首字母搜索请求";
            case 2:
                return "提示框搜索请求";
            case 10:
                return "TopN请求";
            case 20:
                return "Suggest请求";
            case 30:
                return "Hit请求";
            default:
                return null;
        }
    }

    private int getIndexByType(int type) {
        switch (type) {
            case 0:
                return 0;
            case 1:
                return 1;
            case 2:
                return 2;
            case 10:
                return 3;
            case 20:
                return 4;
            case 30:
                return 5;
            default:
                return -1;
        }
    }

    // Convert URL paras to XML document.
    protected Document getParas(String paras) {
        SAXReader reader = new SAXReader();
        Document doc = null;
        InputStream xmlInputStream = null;

        try {
            paras = URLDecoder.decode(paras, "UTF-8");

            xmlInputStream = new ByteArrayInputStream(paras.getBytes("UTF-8"));
            doc = reader.read(xmlInputStream);
        } catch (Exception e) {
            logger.error("Error!The Request Parameter cannot parse to XML");
            return null;
        } finally {
            try {
                if (xmlInputStream != null)
                    xmlInputStream.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        return doc;
    }

    // Gets Calendar by access log file name
    private Calendar getCalendarFromFileName(String file) {
        Calendar c = null;
        String namePattern = "localhost_access_log\\.(\\d{4})-(\\d{2})-(\\d{2})\\.txt";
        Pattern pattern = Pattern.compile(namePattern);
        Matcher matcher = pattern.matcher(file);
        // just get log file that match with pattern
        // "localhost_access_log\\.(\\d{4})-(\\d{2})-(\\d{2})\\.txt"
        if (matcher.matches()) {
            c = Calendar.getInstance();
            int year = Integer.parseInt(matcher.group(1));
            int month = Integer.parseInt(matcher.group(2)) - 1;
            int day = Integer.parseInt(matcher.group(3));
            c.set(year, month, day, 0, 0, 0);
            c.set(Calendar.MILLISECOND, 0);
        }
        return c;
    }

    // Checks string whether match with pattern
    private boolean isMatchPattern(String keyword, String patternStr) {
        Pattern patternOne = Pattern.compile(patternStr);
        return patternOne.matcher(keyword).matches();
    }

    private void convertToMap(List<TopKeyword> list) {
        for (TopKeyword topKeyword : list) {
            allTopKeywordSet.put(topKeyword.getKeyword(), topKeyword.getCount());
        }
    }

    private AccessLog[] getDefaultAccessLogs(Date accessLowerTime, Date accessUpperTime) {
        // Each file will generate six records, so the array's length of
        // AccessLog should be six. Also, in order to increase the execution
        // efficiently, keep each record as follow's rule
        // log.type == 0, saved in the first
        // log.type == 1, saved in the second
        // log.type == 2, saved in the third
        // log.type == 10, saved in the forth
        // log.type == 20, saved in the fifth
        // log.type == 30, saved in the sixth
        AccessLog[] logs = new AccessLog[6];
        logs[0] = getDefaultAccessLog(0, accessLowerTime, accessUpperTime);
        logs[1] = getDefaultAccessLog(1, accessLowerTime, accessUpperTime);
        logs[2] = getDefaultAccessLog(2, accessLowerTime, accessUpperTime);
        logs[3] = getDefaultAccessLog(10, accessLowerTime, accessUpperTime);
        logs[4] = getDefaultAccessLog(20, accessLowerTime, accessUpperTime);
        logs[5] = getDefaultAccessLog(30, accessLowerTime, accessUpperTime);

        return logs;
    }

    private AccessLog getDefaultAccessLog(int type, Date accessLowerTime, Date accessUpperTime) {
        AccessLog log = new AccessLog();
        log.setType(type);
        log.setDescription(getDescriptionByType(type));
        log.setSiteCode(this.siteCode);
        log.setServerNo(this.serverNo);

        log.setLastUpdateTime(new Date());
        log.setAccessLowerTime(accessLowerTime);
        log.setAccessUpperTime(accessUpperTime);
        log.setCount(0);
        return log;
    }

    private class AnalyzerTask implements Runnable {

        private Lock lock = new ReentrantLock();
        private String fileName;
        private CountDownLatch latch;

        public AnalyzerTask(String fileName, CountDownLatch latch) {
            this.fileName = fileName;
            this.latch = latch;
        }

        public void run() {
            logger.debug("Getting access log from file " + fileName);
            Calendar calendar = getCalendarFromFileName(fileName);
            String lineSeparator = System.getProperties().getProperty("line.separator");

            Map<String, AccessLog[]> logsMap = new HashMap<String, AccessLog[]>();
            // Default start time yyyy-MM-dd 00:00:00
            Calendar timer = Calendar.getInstance();
            timer.set(Calendar.YEAR, calendar.get(Calendar.YEAR));
            timer.set(Calendar.MONTH, calendar.get(Calendar.MONTH));
            timer.set(Calendar.DAY_OF_MONTH, calendar.get(Calendar.DAY_OF_MONTH));
            timer.set(Calendar.HOUR_OF_DAY, 0);
            timer.set(Calendar.MINUTE, 0);
            timer.set(Calendar.SECOND, 0);

            Date lastAccessLowerTime = timer.getTime();
            timer.add(Calendar.MINUTE, MINUTE_INTERVAL);
            Date lastAccessUpperTime = timer.getTime();

            // Gets all accesslogs
            SimpleDateFormat date_time_format = new SimpleDateFormat(StringConst.DATEFTIMEFORMAT);
            for (int i = 0; i < MINUTE_SPLIT_TOTAL_COUNT; i++) {
                // Gets default accesslogs
                AccessLog[] logs = getDefaultAccessLogs(lastAccessLowerTime, lastAccessUpperTime);
                logsMap.put(date_time_format.format(lastAccessLowerTime), logs);
                lastAccessLowerTime = timer.getTime();
                timer.add(Calendar.MINUTE, MINUTE_INTERVAL);
                lastAccessUpperTime = timer.getTime();
            }

            RandomAccessFile file = null;
            FileChannel channel = null;

            try {

                String fullPath = logsFolder + "/" + fileName;
                file = new RandomAccessFile(fullPath, "r");
                channel = file.getChannel();

                // Sets buffer size to 0.5M
                ByteBuffer byteBuf = ByteBuffer.allocate(BUFFER_SIZE);

                // Reads 0.5M from file
                StringBuilder lastReadString = new StringBuilder();

                while (channel.read(byteBuf) != -1) {
                    // Gets char from ByteBuffer
                    byteBuf.rewind();
                    byte[] bs = new byte[BUFFER_SIZE];
                    byteBuf.get(bs);
                    String aStringBuffer = new String(bs, 0, BUFFER_SIZE);
                    byteBuf.clear();

                    // Generates a string that needed to be handle.
                    lastReadString.append(aStringBuffer);
                    String handleString = lastReadString.toString();

                    // Gets one line by line.separator
                    int newLineIndex = handleString.indexOf(lineSeparator);
                    while (newLineIndex != -1) {

                        String newLine = handleString.substring(0, newLineIndex);
                        AccessLog accessLog;
                        // Catch any exception for invalid row and ignore it.
                        try {
                            // Gets one record from one line.
                            accessLog = getAccessLog(newLine, calendar.getTime());
                        } catch (Exception e) {
                            accessLog = null;
                            logger.error("Ignore one invalid row " + e);
                            e.printStackTrace();
                        }

                        if (accessLog != null) {
                            Calendar accessTime = Calendar.getInstance();
                            accessTime.setTime(accessLog.getAccessLowerTime());
                            int minute = accessTime.get(Calendar.MINUTE);
                            minute = (minute / MINUTE_INTERVAL) * MINUTE_INTERVAL;
                            accessTime.set(Calendar.MINUTE, minute);
                            accessTime.set(Calendar.SECOND, 0);
                            AccessLog[] logs = logsMap.get(date_time_format.format(accessTime.getTime()));

                            // re-set lower time.
                            accessLog.setAccessLowerTime(accessTime.getTime());

                            // Count access logs for each type.
                            int index = getIndexByType(accessLog.getType());
                            if (index == -1) {
                                // received invalid access log, ignore it
                                logger.error("Invalid access log type received, ignore it");
                            } else {
                                // Checks this type's access log whether is
                                // exist in the list.
                                AccessLog tempLog = logs[index];
                                if (tempLog == null) {
                                    // Doesn't exist
                                    accessLog.setCount(1);
                                    logs[index] = accessLog.clone();
                                } else {
                                    // Existed, update the count
                                    tempLog.setCount(tempLog.getCount() + 1);
                                }
                            }
                        }

                        handleString = handleString.substring(newLineIndex + lineSeparator.length());
                        newLineIndex = handleString.indexOf(lineSeparator);
                    }
                    lastReadString = new StringBuilder(handleString);
                }
            } catch (IOException e) {
                logger.error(String.format("Analyzes file {0} failed", fileName));
                logger.error(e.toString());
                e.printStackTrace();
            } catch (Exception e) {
                logger.error(e);
                e.printStackTrace();
            } finally {
                try {
                    if (file != null)
                        file.close();
                    if (channel != null)
                        channel.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }

                // Add logs to result map.
                try {
                    lock.lock();
                    List<AccessLog[]> list = new ArrayList<AccessLog[]>();
                    list.addAll(logsMap.values());
                    result.put(calendar.getTime(), list);
                } catch (Exception e) {
                    logger.error(e);
                    e.printStackTrace();
                } finally {
                    lock.unlock();
                }

                latch.countDown();
            }
        }
    }
}